{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'prepare/mesolitica-tpu.json'\n",
    "b2_application_key_id = os.environ['b2_application_key_id']\n",
    "b2_application_key = os.environ['b2_application_key']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-tpu-general')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = '1000000'\n",
    "directory = 't5-3x-super-tiny-v3'\n",
    "!rm -rf output out {directory}\n",
    "!mkdir {directory}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.data-00000-of-00002')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.data-00000-of-00002')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.data-00001-of-00002')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.data-00001-of-00002')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.index')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.index')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.meta')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.meta')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/checkpoint')\n",
    "blob.download_to_filename(f'{directory}/checkpoint')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/operative_config.gin')\n",
    "blob.download_to_filename(f'{directory}/operative_config.gin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{directory}/checkpoint', 'w') as fopen:\n",
    "    fopen.write(f'model_checkpoint_path: \"model.ckpt-{model}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from b2sdk.v1 import *\n",
    "info = InMemoryAccountInfo()\n",
    "b2_api = B2Api(info)\n",
    "application_key_id = b2_application_key_id\n",
    "application_key = b2_application_key\n",
    "b2_api.authorize_account(\"production\", application_key_id, application_key)\n",
    "file_info = {'how': 'good-file'}\n",
    "b2_bucket = b2_api.get_bucket_by_name('malaya-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<b2sdk.file_version.FileVersionInfo at 0x7ff59f758780>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tar = 't5-3x-super-tiny-2021-07-28.tar.gz'\n",
    "os.system(f'tar -czvf {tar} {directory}')\n",
    "outPutname = f'pretrained/{tar}'\n",
    "b2_bucket.upload_local_file(\n",
    "    local_file=tar,\n",
    "    file_name=outPutname,\n",
    "    file_infos=file_info,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f'rm {tar}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install t5==0.5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = t5.models.MtfModel(\n",
    "    model_dir=directory,\n",
    "    tpu=None,\n",
    "    tpu_topology=None,\n",
    "    model_parallelism=1,\n",
    "    batch_size=1,\n",
    "    sequence_length={\"inputs\": 512, \"targets\": 512},\n",
    "    learning_rate_schedule=0.003,\n",
    "    save_checkpoints_steps=5000,\n",
    "    keep_checkpoint_max=3,\n",
    "    iterations_per_loop=100,\n",
    "    mesh_shape=\"model:1,batch:1\", \n",
    "    mesh_devices=[\"cpu:0\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = \"\"\"\n",
    "Amanah Kedah berpendapat jika ada Adun Pakatan Harapan atau Bersatu negeri itu mahu berpaling tadah memberikan sokongan kepada kumpulan Muafakat Nasional, mereka perku membuat kenyataan rasmi mengenainya.\n",
    "\n",
    "Pengerusi Amanah Kedah, Phahrolrazi Mohd Zawawi, berkata disebabkan tiada mana-mana Adun membuat kenyataan berhubung isu itu maka kerajaan negeri berpendapat tiada apa-apa yang berlaku.\n",
    "\n",
    "Ditemui media selepas mengadakan pertemuan tertutup lebih sejam dengan Menteri Besar, Mukhriz Mahathir, hari ini Phahrolrazi berkata pihaknya juga mendapati kerajaan negeri masih berfungsi seperti biasa.\n",
    "\n",
    "\"Kami bincang keadaan semasa, ada juga kita sentuh (cubaan menukar kerajaan negeri), tetapi kita lihat kerajaan masih berfungsi.\n",
    "\n",
    "\"Tidak ada apa-apa kenyataan dari pihak sana (pembangkang) bahawa mereka sudah cukup majoriti setakat ini,\" katanya seperti dipetik BH Online.\n",
    "\n",
    "Spekulasi mengenai pertukaran kerajaan menjadi kencang sejak semalam ekoran berlaku pertemuan tertutup pemimpin PAS dan Umno Kedah di Alor Setar semalam.\n",
    "\n",
    "Turut hadir Setiausaha Agung PAS yang juga Menteri di Jabatan Perdana Menteri, Takiyuddin Hassan, dan Menteri Besar Terengganu, Dr Ahmad Samsuri Mokhtar.  \n",
    "\n",
    "Cuba jatuhkan sejak dulu\n",
    "\n",
    "Perkembangan itu berlaku kesan tindakan PKR memecat dan menggantung sejumlah anggota mereka baru-baru ini dan dipercayai memberi kesan terhadap pendirian wakil rakyat parti itu di Kedah.\n",
    "\n",
    "Turut disebut-sebut akan beralih arah dalam perjalanan politik mereka ialah Adun Bersatu.\n",
    "\n",
    "Untuk rekod berdasarkan pecahan parti PAS menguasai kerusi terbesar dalam DUN dan lazimnya pemimpin parti itu akan menjadi pilihan menjadi menteri besar jika berlaku pertukaran kerajaan.\n",
    "\n",
    "Menurut Phahrolrazi, jika ada mana-mana wakil rakyat Bersatu atau PH mahu melompat, mereka wajar menyatakannya secara rasmi.\n",
    "\n",
    "Tanpa kenyataan begitu, katanya, Amanah beranggapan isu perubahan kerajaan negeri masih bersifat spekulasi.\n",
    "\n",
    "Timbalan Pengerusi Amanah Kedah, Dr Ismail Salleh, pula berkata ada kemungkinan Adun Bersatu, PH atau exco negeri tu yang sudah diumpan untuk membelakangkan mandat rakyat.\n",
    "\n",
    "Beliau yang juga exco Kedah berkata memang sejak dulu lagi PAS cuba menjatuhkan kerajaan negeri dengan memujuk Adun PH serta Bersatu bertindak seperti rakan mereka di Perak, Johor dan Selangor.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://harakahdaily.net/index.php/2020/05/22/apa-ph-buat-isu-kenaikan-harga-barang/\n",
    "\n",
    "string2 = \"\"\"\n",
    "SEKARANG PH tengah mainkan isu harga barang untuk memburukkan Kerajaan PN. Kata Ahli Majlis Setiausaha PH (Saifuddin Nasution, Khalid Samad dan Ong Kian Ming) harga barang mentah dan basah menunjukkan ada peningkatan di banyak tempat. Ini membebankan rakyat kata mereka.\n",
    "\n",
    "Saya setuju, Kerajaan PN perlu pantau dan ambil apa-apa tindakan yang perlu jika ada peningkatan harga barang basah atau mentah.\n",
    "\n",
    "Tapi dalam masa yang sama saya fikir elok juga kalau kita imbas balik apa pula tindakan yang diambil oleh Kerajaan PH ketika rakyat tertekan dengan harga barang yang melonjak gila-gila masa mereka jadi Kerajaan Persekutuan selama 22 bulan dulu.\n",
    "\n",
    "Mari kita semak balik.\n",
    "\n",
    "(1) Kenaikan harga telur\n",
    "Saifuddin Nasution – “Kementerian akan siasat.”\n",
    "Salahuddin Ayub – “Ianya adalah perkara biasa disebabkan kenaikan bahan makanan ayam.”\n",
    "\n",
    "(2) Kenaikan harga bawang 3 kali ganda\n",
    "Saifuddin Nasution – “KPNHEP akan pantau kenaikan ini. Rakyat perlu tukar kepada bawang Holland.”\n",
    "\n",
    "(3) Kenaikan harga ikan kembung\n",
    "Salahuddin Ayub – “Saya pelik mengapa harga ikan kembung naik sedangkan ada 7 bilion ikan kembung dalam laut.”\n",
    "\n",
    "(4) Kenaikan harga barangan kawalan\n",
    "Saifuddin Nasution – “KPNHEP tidak menerima sebarang aduan kenaikan harga barang kawalan. Kita akan siasat.”\n",
    "\n",
    "(5) Kenaikan harga barang mentah dan basah menjelang Aidilfitri 2019\n",
    "Saifuddin Nasution – “KPNHEP puas hati dengan trend semasa harga barang keperluan. Tiada kenaikan mendadak (kenaikan sedikit demi sedikit itu perkara biasa).”\n",
    "\n",
    "(6) Kenaikan harga barang keperluan pada penghujung 2018 dan sepanjang 2019\n",
    "Lim Guan Eng – “Kenaikan harga barang disebakan banyak faktor seperti penyusutan nilai Ringgit, musim perayaan, kegiatan penyeludupan, kemarau di negara pengeluar selain sikap peniaga yang mahu mengaut keuntungan berlebihan. Ianya tidak dapat dielakkan.”\n",
    "\n",
    "Yang saya listkan di atas hanya sebahagian kecil daripada kenyataan Menteri-menteri PH. Ada banyak lagi.\n",
    "\n",
    "Kesimpulannya, PH tak buat apa-apa pun berkaitan kenaikan harga barang keperluan yang meresahkan rakyat semasa mereka jadi Kerajaan. Tapi sekarang tiba-tiba pula mereka jadi prihatin dan mahu jadi hero rakyat.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.malaysiakini.com/news/526581\n",
    "\n",
    "string3 = \"\"\"\n",
    "KATA MP | Saya membaca kenyataan beberapa orang penganalisis politik seperti yang dilaporkan dalam media - misalnya di Malaysiakini - mengenai perkembangan semasa politik dalam negara.\n",
    "\n",
    "Mereka, antara lain, menyarankan agar Pakatan Harapan (PH) memberi tumpuan kepada PRU15 untuk mengambil kerajaan dan bukannya melakukan hal itu pada masa sekarang.\n",
    "\n",
    "Seorang pemimpin Umno pula dilaporkan oleh akhbar Sinar Harian dengan bangga menggunakan ucapan titah yang di-Pertuan Agong semasa hari perasmian sidang parlimen pada 18 Mei lalu bagi menunjukkan kononnya kerajaan yang ada sekarang ialah \"kerajaan Tuanku\".\n",
    "\n",
    "Pertamanya, saya ingin ucapkan terima kasihlah kepada para penganalisis politik tersebut atas analisis mereka. Maaf - dan dengan penuh hormat - jika saya katakan bahawa analisis anda semua bernilai amat amat rendah dan hampir mencapai tahap tidak ada nilai.\n",
    "\n",
    "Saya tidak pasti di mana anda semua simpan disiplin ilmu anda.\n",
    "\n",
    "Keduanya, sekiranya kritikan anda semua itu benar-benar jujur, cadangan anda semua (bahawa perlu tunggu hingga PRU15 untuk ambil alih kerajaan) itu sewajarnya ditujukan kepada Perikatan Nasional (PN) yang membentuk kerajaan sekarang tanpa sebarang mandat rakyat dalam sebarang pilihan raya.\n",
    "\n",
    "Ia diakui sendiri oleh Perdana Menteri Muhyiddin Yassin semasa berucap secara langsung di televisyen baru-baru ini. Bagi saya ia sesuai dinobatkan sebagai Ucapan Mak Cik Kiah.\n",
    "\n",
    "Dibentuk secara sah, bermoral\n",
    "\n",
    "Ketiga, jika PH kembali berkuasa pada masa ini, hakikatnya ia berkuasa semula dengan mandat rakyat yang diberi oleh rakyat menerusi PRU14.\n",
    "\n",
    "Mandat tersebut tidak pernah ditarik balik! Bukankah mandat itu untuk tempoh lima tahun? Justeru, siapa yang tidak hormat kepada mandat rakyat tersebut?\n",
    "\n",
    "Keempat, meminta agar PH menunggu sehingga PRU15 nampaknya berdasarkan andaian atau premis bahawa kerajaan PN ini wajar dibiarkan mentadbir beberapa tahun lagi.\n",
    "\n",
    "Pun begitu ada beberapa fakta yang perlu diperhatikan, iaitu:\n",
    "\n",
    "Ia merupakan kerajaan yang memiliki saiz kabinat yang lebih besar berbanding PH.\n",
    "Kerajaan PN meriah mengagih-agihkan ghanimah kepada aktor-aktor politik seperti pelantikan sebagai duta khas bertaraf menteri dan pelantikan dalam syarikat berkaitan kerajaan (GLC) bagi 'membeli' kesetiaan\n",
    "Ia merupakan kerajaan yang lebih teruja 'membebaskan' insan yang terpalit dengan skandal mega 1MDB.\n",
    "Sedarkah para penganalisis berkenaan mengenai semua realiti ini, selain banyak isu lain membabitkan imej negara, ekonomi, pelaburan asing, keyakinan pelabur dan sebagainya?\n",
    "\n",
    "Kelima, kerajaan PH yang ditumbangkan oleh PN sekarang adalah kerajaan yang tidak ada sedikit pun 'daki' ketakabsahan (legitimacy) sama ada dari sudut perlembagaan, proses demokrasi atau dari sudut moral dan agama.\n",
    "\n",
    "Di mana suara mereka sebelum ini\n",
    "\n",
    "Tiada sedikit pun elemen pengkhianatan dalam pembentukan kerajaan PH. Ia dibentuk secara sah, secara bermoral dan beretika sambil menjunjung tinggi semangat dan ruh Perlembagaan Persekutuan.\n",
    "\n",
    "Ia adalah sepenuhnya kerajaan rakyat yang mewakili semua bangsa, etnik dan agama.\n",
    "\n",
    "Ya, benar, kerajaan PH itu tidaklah sempurna. Ada kelemahan di sini sana. Namun siapa yang boleh menafikan kesungguhan kerajaan PH bagi membetulkan kerosakan amat teruk yang ditinggalkan oleh kerajaan sebelum ini?\n",
    "\n",
    "Kesungguhan itu boleh dilihat oleh ramai orang di dalam atau di luar negara. Semasa saya bertugas di luar negara atas kapasiti anggota pentadbiran kerajaan PH, ramai rakan sejawat saya di luar yang bukan sahaja memuji kita malah lebih daripada itu mereka berjanji untuk membantu.\n",
    "\n",
    "Saya masih ingat mantan perdana menteri - Dr Mahathir Mohamad - sering mengatakan bahawa hanya setelah beberapa bulan PH mengambil alih tampuk pemerintahan negara, usaha menukar imej negara ini yang dikenali sebagai negara kleptokrat mula menunjukkan hasilnya.\n",
    "\n",
    "Keenam, saya tertanya-tanya di mana suara para penganalisis politik itu semua ketika PN mengambil alih kerajaan PH secara tidak bermaruah hingga digelar kerajaan pintu belakang atau kerajaan tebuk atap?\n",
    "\n",
    "Ya, di mana suara mereka ketika Langkah Sheraton digerakkan?\n",
    "\n",
    "Kenapa - pada masa itu - kita tidak dengar mereka menasihati PN agar menunggu sehingga PRU15 untuk menjadi kerajaan?\n",
    "\n",
    "Tentang isu \"kerajaan Tuanku\" atau \"kerajaan beta\" itu, maaflah, takkanlah serendah itu tahap politik mereka? Kenapa terlalu paranoid dengan lafaz itu?\n",
    "\n",
    "Dalam sistem demokrasi ala Westminster yang menjunjung prinsip raja berperlembagaan, tiada sebarang keanehan pun dengan istilah tersebut. Ia fenomena yang lumrah. Saya rasa aneh apabila ada yang sanggup mengeksploitasi istilah berkenaan.\n",
    "\n",
    "Amat tidak wajar dan amat tidak beretika sama sekali untuk mengheret institusi diraja - sama seperti tidak wajar mengeksploitasi isu agama dan perkauman - demi kelangsungan jangka hayat politik yang kita tahu sedang begitu nazak sekarang.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "string4 = \"\"\"\n",
    "ORANG ramai tidak perlu panik atau bimbang dengan berita tular di media sosial mendakwa ayam yang dibeli di pasar raya menjadi `ejen' pembawa virus Covid-19 dan boleh membahayakan nyawa.\n",
    "\n",
    "Ini berdasarkan kenyataan dikeluarkan Jabatan Perkhidmatan Veteriner Putrajaya (JPVP) pada 9 Mei lalu, menjelaskan bahawa ayam ternakan tidak boleh dijangkiti virus Covid-19, walaupun pekerja kilang berkenaan atau mereka yang mengurus atau memproses ayam mempunyai kontak Covid-19.\n",
    "\n",
    "Jurucakap jabatan itu berkata, virus covid-19 juga tidak mampu bertahan lama pada daging ayam sekiranya pekerja syarikat ayam terbabit disahkan positif Covid-19.\n",
    "\n",
    "Katanya, dalam proses penyembelihan dan pemprosesan yang dilakukan terhadap ayam, virus Covid-19 yang mungkin ada pada ayam akan mati dan ayam berkenaan selamat dimakan.\n",
    "\n",
    "\"Kenyataan ini juga disokong dengan laporan dikeluarkan Pertubuhan Kesihatan Sedunia (WHO) pada 7 April 2020, yang menjelaskan bahawa makanan yang diproses selamat dimakan.\n",
    "\n",
    "\"Malah, laporan itu juga mengesahkan bahawa ujian Covid-19 yang dibuat terhadap ayam dari negara China menunjukkan semuanya negatif penyakit berkenaan,\" katanya di sini, tadi.\n",
    "\n",
    "Isu kebimbangan orang ramai mengenai kedudukan ayam timbul selepas wujud kluster baharu Covid-19 di sebuah kilang pemprosesan ayam di Pedas Rembau Negeri Sembilan, pada 2 Mei lalu.\n",
    "\n",
    "Kebimbangan itu semakin bertambah apabila isu berkenaan viral dalam laman sosial hingga mendapat pelbagai pandangan daripada netizen.\n",
    "\n",
    "Dalam laporan yang dikeluarkan Kementerian Kesihatan Malaysia (KKM) peringkat awal mendapati 60 pekerja loji pemprosesan ayam di kilang terbabit positif Covid-19.\n",
    "\n",
    "Difahamkan kilang berkenaan adalah loji pemprosesan ayam komersial di bawah seliaan JPV dan dikatakan memproses dan membekalkan sehingga 54,000 ekor ayam sehari daripada jumlah keseluruhan bekalan ayam segar seluruh negara purata sebanyak 1.6 juta ekor sehari.\n",
    "\n",
    "Sumber ayam hidup diperoleh daripada 79 ladang ayam kontrak di bawah kawal selia syarikat dan 23 ladang persendirian.\n",
    "\n",
    "Syarikat itu juga membekalkan ayam yang telah diproses ke beberapa pasar raya.\n",
    "\n",
    "Kluster baharu Covid-19 di kilang berkenaan mula dikesan apabila seorang kakitangan syarikat mengalami gejala demam, batuk, selsema, sesak nafas dan sakit kepala pada 5 April lalu.\n",
    "\n",
    "Pekerja terbabit telah menjalani saringan ujian Covid-19 pada 10 April dan keputusan ujian disahkan positif dua hari kemudian.\n",
    "\n",
    "Sehingga 8 Mei lalu, seramai 786 orang kakitangan dan ahli keluarga kilang berkenaan telah dijalankan saringan ujian Covid-19 dan daripada jumlah terbabit, 60 didapati positif Covid-19.\n",
    "\n",
    "Seramai 286 negatif dan 440 orang masih menunggu keputusan dan pekerja yang disahkan popsitif Covid-19 telah dikuarantin.\n",
    "\n",
    "Susulan itu, KKM mengeluarkan arahan menghentikan operasi kilang terbabit selama 14 hari berkuatkuasa dari 7 hingga 20 Mei ini.\n",
    "\n",
    "Dalam laporan yang dikeluarkan KKM semalam, terdapat pertambahan 43 kes, membabitkan warga pekerja kilang berkenan menjadikan jumlah kes positif Covid-19 kepada 131 kes.\n",
    "\n",
    "\"Daripada jumlah itu 122 kes melibatkan bukan warganegara,\" kata Ketua Pengarah Kesihatan, Dr Noor Hisham Abdullah.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# minimum cleaning, just simply to remove newlines.\n",
    "def cleaning(string):\n",
    "    string = string.replace('\\n', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string\n",
    "\n",
    "string = cleaning(string)\n",
    "string2 = cleaning(string2)\n",
    "string3 = cleaning(string3)\n",
    "string4 = cleaning(string4)\n",
    "\n",
    "# with tf.io.gfile.GFile('test.txt', \"w\") as f:\n",
    "#     f.write(\"ringkasan: %s\\n\" % string2)\n",
    "#     f.write(\"ringkasan: %s\\n\" % string3)\n",
    "#     f.write(\"ringkasan: %s\\n\" % string4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf output/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 't5-3x-super-tiny-v3', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "isolate_session_state: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': None, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7ff442b22128>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n"
     ]
    }
   ],
   "source": [
    "import gin\n",
    "\n",
    "from t5.data import sentencepiece_vocabulary\n",
    "\n",
    "DEFAULT_SPM_PATH = 'prepare/sp10m.cased.ms-en.model'\n",
    "DEFAULT_EXTRA_IDS = 100\n",
    "model_dir = directory\n",
    "\n",
    "def get_default_vocabulary():\n",
    "    return sentencepiece_vocabulary.SentencePieceVocabulary(\n",
    "      DEFAULT_SPM_PATH, DEFAULT_EXTRA_IDS)\n",
    "\n",
    "with gin.unlock_config():\n",
    "    gin.parse_config_file(t5.models.mtf_model._operative_config_path(model_dir))\n",
    "    gin.bind_parameter(\"Bitransformer.decode.beam_size\", 1)\n",
    "    gin.bind_parameter(\"Bitransformer.decode.temperature\", 0)\n",
    "    gin.bind_parameter(\"utils.get_variable_dtype.slice_dtype\", \"float32\")\n",
    "    gin.bind_parameter(\n",
    "        \"utils.get_variable_dtype.activation_dtype\", \"float32\")\n",
    "    \n",
    "vocabulary = t5.data.SentencePieceVocabulary(DEFAULT_SPM_PATH)\n",
    "estimator = model.estimator(vocabulary, disable_tpu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000000, 'model.ckpt-1000000', 't5-3x-super-tiny-v3/model.ckpt-1000000')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "checkpoint_step = t5.models.mtf_model._get_latest_checkpoint_from_dir(model_dir)\n",
    "model_ckpt = \"model.ckpt-\" + str(checkpoint_step)\n",
    "checkpoint_path = os.path.join(model_dir, model_ckpt)\n",
    "checkpoint_step, model_ckpt, checkpoint_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running infer on CPU\n",
      "INFO:tensorflow:feature inputs : Tensor(\"Reshape:0\", shape=(1, 512), dtype=int32)\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/mesh_tensorflow/transformer/utils.py:427: Print (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2018-08-20.\n",
      "Instructions for updating:\n",
      "Use tf.print instead of tf.Print. Note that tf.print returns a no-output operator that directly prints the output. Outside of defuns or eager mode, this operator will not be executed unless it is directly specified in session.run or used as a control dependency for other operators. This is only a concern in graph mode. Below is an example of how to ensure tf.print executes in graph mode:\n",
      "\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 24576        slice_size 24576        Shape[heads=384, d_model=64]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias size 192          slice_size 192          Shape[heads=6, buckets=32]                                  \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/layer_norm/scale                 size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 24576        slice_size 24576        Shape[heads=384, d_model=64]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/layer_norm/scale                 size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16384        slice_size 16384        Shape[d_model=64, d_ff=256]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16384        slice_size 16384        Shape[d_ff=256, d_model=64]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/layer_norm/scale                 size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable decoder/final_layer_norm/scale                               size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 24576        slice_size 24576        Shape[heads=384, d_model=64]                                \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 192          slice_size 192          Shape[heads=6, buckets=32]                                  \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 24576        slice_size 24576        Shape[d_model=64, heads=384]                                \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/layer_norm/scale                 size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16384        slice_size 16384        Shape[d_model=64, d_ff=256]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16384        slice_size 16384        Shape[d_ff=256, d_model=64]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/layer_norm/scale                 size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable encoder/final_layer_norm/scale                               size 64           slice_size 64           Shape[d_model=64]                                           \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 2056192      slice_size 2056192      Shape[vocab=32128, d_model=64]                              \n",
      "INFO:tensorflow:Trainable Variables            count: 26      Total size: 2417472          Total slice_size: 2417472        \n",
      "INFO:tensorflow:All Variables                  count: 26      Total size: 2417472          Total slice_size: 2417472        \n",
      "INFO:tensorflow:Counters:\n",
      "einsum: 4.06e+09\n",
      "einsum_unique: 4.06e+09\n",
      "output: 1.3e+08\n",
      " output/AddOperation: 1.88e+07\n",
      " output/BinaryOpWithBroadcasting: 1.31e+06\n",
      " output/Constant: 3.94e+05\n",
      " output/EinsumOperation: 2.96e+07\n",
      " output/ImportOperation: 524\n",
      " output/MinMaxOperation: 7.87e+05\n",
      " output/OneHotOperation: 4.97e+07\n",
      " output/RangeOperation: 1.02e+03\n",
      " output/ReduceOperation: 2.2e+04\n",
      " output/ReshapeOperation: 3.05e+06\n",
      " output/ScalarAddOperation: 1.05e+06\n",
      " output/ScalarMultiplyOperation: 2.4e+06\n",
      " output/ShiftOperation: 512\n",
      " output/SlicewiseOperation: 1.52e+07\n",
      " output/StopGradient: 4.72e+06\n",
      " output/Variable: 2.42e+06\n",
      " output/WhileLoopOperation: 3.94e+05\n",
      "output_unique: 1.3e+08\n",
      " output_unique/AddOperation: 1.88e+07\n",
      " output_unique/BinaryOpWithBroadcasting: 1.31e+06\n",
      " output_unique/Constant: 3.94e+05\n",
      " output_unique/EinsumOperation: 2.96e+07\n",
      " output_unique/ImportOperation: 524\n",
      " output_unique/MinMaxOperation: 7.87e+05\n",
      " output_unique/OneHotOperation: 4.97e+07\n",
      " output_unique/RangeOperation: 1.02e+03\n",
      " output_unique/ReduceOperation: 2.2e+04\n",
      " output_unique/ReshapeOperation: 3.05e+06\n",
      " output_unique/ScalarAddOperation: 1.05e+06\n",
      " output_unique/ScalarMultiplyOperation: 2.4e+06\n",
      " output_unique/ShiftOperation: 512\n",
      " output_unique/SlicewiseOperation: 1.52e+07\n",
      " output_unique/StopGradient: 4.72e+06\n",
      " output_unique/Variable: 2.42e+06\n",
      " output_unique/WhileLoopOperation: 3.94e+05\n",
      "variables: 2.42e+06\n",
      " variables/trainable: 2.42e+06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1475: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from t5-3x-super-tiny-v3/model.ckpt-1000000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: output/temp-b'1637670827'/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "from mesh_tensorflow.transformer import dataset as transformer_dataset\n",
    "\n",
    "def serving_input_fn():\n",
    "    inputs = tf.placeholder(\n",
    "            dtype=tf.string,\n",
    "            shape=[None],\n",
    "            name=\"inputs\")\n",
    "\n",
    "    batch_size = tf.shape(inputs)[0]\n",
    "    padded_inputs = tf.pad(inputs, [(0, tf.mod(-tf.size(inputs), batch_size))])\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(padded_inputs)\n",
    "    dataset = dataset.map(lambda x: {\"inputs\": x})\n",
    "    dataset = transformer_dataset.encode_all_features(dataset, vocabulary)\n",
    "    dataset = transformer_dataset.pack_or_pad(\n",
    "        dataset=dataset,\n",
    "        length=model._sequence_length,\n",
    "        pack=False,\n",
    "        feature_keys=[\"inputs\"]\n",
    "    )\n",
    "    dataset = dataset.batch(tf.cast(batch_size, tf.int64))\n",
    "    features = tf.data.experimental.get_single_element(dataset)\n",
    "    return tf.estimator.export.ServingInputReceiver(\n",
    "        features=features, receiver_tensors=inputs)\n",
    "\n",
    "out = estimator.export_saved_model('output', serving_input_fn, checkpoint_path=checkpoint_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-21-5b89b6a20c22>:7: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "INFO:tensorflow:Restoring parameters from output/1637670827/variables/variables\n"
     ]
    }
   ],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config = config)\n",
    "meta_graph_def = tf.saved_model.loader.load(\n",
    "        sess,\n",
    "        [tf.saved_model.tag_constants.SERVING],\n",
    "        out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3x-super-tiny/model.ckpt'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, '3x-super-tiny/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = [\n",
    "    n.name\n",
    "    for n in tf.get_default_graph().as_graph_def().node\n",
    "    if ('encoder' in n.op\n",
    "    or 'decoder' in n.name\n",
    "    or 'shared' in n.name\n",
    "    or 'inputs' in n.name\n",
    "    or 'output' in n.name\n",
    "    or 'SentenceTokenizer' in n.name\n",
    "    or 'self/Softmax' in n.name)\n",
    "    and 'adam' not in n.name\n",
    "    and 'Assign' not in n.name\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names,\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from 3x-super-tiny/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-24-504c79665720>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 42 variables.\n",
      "INFO:tensorflow:Converted 42 variables to const ops.\n",
      "1504 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('3x-super-tiny', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import struct\n",
    "\n",
    "unknown = b'\\xff\\xff\\xff\\xff'\n",
    "\n",
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "        \n",
    "    for node in graph_def.node:\n",
    "        \n",
    "        if node.op == 'RefSwitch':\n",
    "          node.op = 'Switch'\n",
    "          for index in xrange(len(node.input)):\n",
    "            if 'moving_' in node.input[index]:\n",
    "              node.input[index] = node.input[index] + '/read'\n",
    "        elif node.op == 'AssignSub':\n",
    "          node.op = 'Sub'\n",
    "          if 'use_locking' in node.attr: del node.attr['use_locking']\n",
    "        elif node.op == 'AssignAdd':\n",
    "          node.op = 'Add'\n",
    "          if 'use_locking' in node.attr: del node.attr['use_locking']\n",
    "        elif node.op == 'Assign':\n",
    "          node.op = 'Identity'\n",
    "          if 'use_locking' in node.attr: del node.attr['use_locking']\n",
    "          if 'validate_shape' in node.attr: del node.attr['validate_shape']\n",
    "          if len(node.input) == 2:\n",
    "            node.input[0] = node.input[1]\n",
    "            del node.input[1]\n",
    "            \n",
    "        if 'Reshape/shape' in node.name or 'Reshape_1/shape' in node.name:\n",
    "            b = node.attr['value'].tensor.tensor_content\n",
    "            arr_int = [int.from_bytes(b[i:i + 4], 'little') for i in range(0, len(b), 4)]\n",
    "            if len(arr_int):\n",
    "                arr_byte = [unknown] + [struct.pack('<i', i) for i in arr_int[1:]]\n",
    "                arr_byte = b''.join(arr_byte)\n",
    "                node.attr['value'].tensor.tensor_content = arr_byte\n",
    "            \n",
    "            if len(node.attr['value'].tensor.int_val):\n",
    "                node.attr['value'].tensor.int_val[0] = -1\n",
    "    \n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('3x-super-tiny/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'import/inputs:0' shape=(?,) dtype=string>,\n",
       " <tf.Tensor 'import/SelectV2_3:0' shape=(?, 512) dtype=int32>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = g.get_tensor_by_name('import/inputs:0')\n",
    "o = g.get_tensor_by_name('import/SelectV2_3:0')\n",
    "i, o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sess = tf.Session(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 29s, sys: 13.3 s, total: 1min 42s\n",
      "Wall time: 4.77 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(15, 512)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "o_ = test_sess.run(o, feed_dict = {i: [f'ringkasan: {string}', \n",
    "                                       f'ringkasan: {string2}', \n",
    "                                       f'ringkasan: {string3}',\n",
    "                                       f'ringkasan: {string4}',\n",
    "                                       f'tajuk: {string}', \n",
    "                                       f'tajuk: {string2}', \n",
    "                                       f'tajuk: {string3}',\n",
    "                                       f'tajuk: {string4}',\n",
    "                                       'terjemah Melayu ke Inggeris: Speaker Dewan Rakyat Azhar Harun menegur ahli-ahli parlimen pembangkang selepas kira-kira sejam mereka menyatakan rasa tidak puas hati pada persidangan khas parlimen hari ini.',\n",
    "                                       \"grafik pengetahuan: Yang Berhormat Dato' Sri Haji Mohammad Najib bin Tun Haji Abdul Razak ialah ahli politik Malaysia dan merupakan bekas Perdana Menteri Malaysia ke-6, yang mana beliau menjawat jawatan dari 3 April 2009 hingga 9 Mei 2018. Beliau juga pernah berkhidmat sebagai bekas Menteri Kewangan dan merupakan Ahli Parlimen Pekan, Pahang. Beliau merupakan anak lelaki sulung kepada bekas Perdana Menteri Malaysia yang kedua, Tun Abdul Razak Dato' Hussein. Di peringkat akar umbi politik, beliau merupakan Presiden UMNO yang ke-7.\",\n",
    "                                       'grafik pengetahuan: Speaker Dewan Rakyat Azhar Harun menegur ahli-ahli parlimen pembangkang selepas kira-kira sejam mereka menyatakan rasa tidak puas hati pada persidangan khas parlimen hari ini.',\n",
    "                                       'grafik pengetahuan: Keuskupan Agung Katolik Rom Maracaibo terletak di barat daya Keuskupan Katolik Rom Machiques.',\n",
    "                                       'parafrasa: Speaker Dewan Rakyat Azhar Harun menegur ahli-ahli parlimen pembangkang selepas kira-kira sejam mereka menyatakan rasa tidak puas hati pada persidangan khas parlimen hari ini.',\n",
    "                                       'terjemah Inggeris ke Melayu: PETALING JAYA: Former prime minister Najib Razak has questioned whether the government knows how to manage the Covid-19 pandemic, outlining several seemingly contradictory announcements it has made.',\n",
    "                                       'terjemah Melayu ke Inggeris: PETALING JAYA: Pertemuan bekas Perdana Menteri, Datuk Seri Najib Tun Razak dan Timbalan Perdana Menteri, Datuk Seri Ismail Sabri Yaakob hari ini adalah bagi membincangkan isu berkaitan hala tuju dan dasar negara.']})\n",
    "o_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sentencepiece as spm\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load(DEFAULT_SPM_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Pengerusi Amanah Kedah, Phahrolrazi Mohd Zawawi berkata, tidak ada apa-apa yang berlaku. 'Kami bincang keadaan semasa, ada juga kita sentuh (cubaan menukar kerajaan negeri).\n",
      "1 PN tidak perlu tukar kepada bawang Belanda. 'KPNHEP akan pantau kenaikan harga barang mentah dan basah. ' ...........................\n",
      "2 ' yang mempunyai saiz kabinat yang lebih besar berbanding PH. Kerajaan 'membeli' kesetiaan adalah kerajaan yang tidak akan menjadi satu- yang tidak ada. ' kata menteri-- dan dengan penuh hormat' kepada '. '...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
      "3 'ejen' pembawa virus Covid-19 dan boleh membahayakan nyawa. '. ' kata laporan dikeluarkan oleh 'ejen', kata '. ' kata laporan dikeluarkan oleh KKM.\n",
      "4 Amanah Kedah rasa jangan tadah\n",
      "5 'KPNHEP perlu pantau kenaikan harga barang\n",
      "6 ' 'daki' ketakabsahan' insan yang 's' - 'daki' ketakabsahan'\n",
      "7 000 ayam ditagih ayam\n",
      "8 Speaker Speaker Speaker Azhar Harun had criticized the opposition MPs after the issue of a few hours to say that dissatisfaction in the conference of the committee of the parliamentary conference today.\n",
      "9 Yang Berhormat Dato' Sri Haji Mohammad Najib bin Tun Haji Abdul Razak member of 3 April 2009, Member of the 6th Member of the Parliament of the Parliament of the Youngist, Member of the 14th Malaysia, Member of the 6th 20th Minister of Malaysia, sex or gender male, educated at 14th Pahang, position held Member of the 14th Parliament of the Young Parliament, noble title former Prime Minister of Malaysia, father Prime Minister of Malaysia, position held former Minister of Finance, member of the 7th Premier of Malaysia.\n",
      "10 Azhar Harun 's ( politician ) parliamentary term librian parliamentary party, Member of the Legislative Assembly of the Parliament of the opposition parliamentary group.\n",
      "11 Roman Catholic Archdiocese of Machiques country of citizenship Roman Catholic Archdiocese of Machiques.\n",
      "12 Speaker Dewan Rakyat Azhar Harun menegur ahli-ahli-ahli parlimen pembangkang selepas kira-kira sejam mereka menyatakan rasa tidak puas hati pada persidangan khas parlimen.\n",
      "13 PETALING JAYA: Bekas Perdana Menteri Najib Razak telah mempersoalkan sama ada kerajaan tahu bagaimana menguruskan Covid-19 pandemic, keluar beberapa yang nampaknya bercanggah pengumuman yang dibuat.\n",
      "14 PETALING JAYA: The Prime Minister Datuk Seri Najib Tun Razak and Deputy Prime Minister Datuk Seri Ismail Sabri Yaakob today is to discuss the issue related to the situation and the national policy.\n"
     ]
    }
   ],
   "source": [
    "for k in range(len(o_)):\n",
    "    print(k, sp_model.DecodeIds(o_[k].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(minimum_size=1536000)',\n",
    "             #'quantize_weights(fallback_min=-10240, fallback_max=10240)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-36-4b799c067114>:3: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "pb = '3x-super-tiny/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "    \n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "       ['inputs'],\n",
    "       ['SelectV2_3'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
